#!/usr/bin/python

# visualize perhost information, including:
# total file read bytes per task
# total file write bytes per task
# total HDFS read bytes per task
# total shuffle bytes per task
# total map output bytes per task

import sys
import re

def getvalue(a):
	return a[a.find("=")+1:].strip("\"")

def getbytevalue(a):
	return int(a[a.rfind("(")+1:].strip(")"))

def gethostid(a):
	b = int(a[a.find("cloud")+5])
	c = int(a[a.find("cloud")+7:a.rfind("\\")])
	host_per_rack = 50
	return (((b-1)*host_per_rack)+c)

def gettaskid(a):
	A = a.split("_")
	return int(A[4])

if len(sys.argv) <= 1:
	print "file name?\n"
	sys.exit(-1)

# precess log, get time break down information 
mhosts = {}
rhosts = {}
for l in open(sys.argv[1]):
	A = l.split()
	if A[0] in ["MapAttempt", "ReduceAttempt"]:
	   	type = {}
		status = {}
		id = 0
		counter = {}
		file_read = 0
		file_write = 0
		hdfs_read = 0
		reduce_shuffle = 0
		map_out = 0
		map_in = 0
	   	for a in A:
		   	# read task type
		   	if a.startswith("TASK_TYPE"):
			   	type = getvalue(a)

		   	# get task id
		   	if a.startswith("TASKID"):
			   	id = gettaskid(getvalue(a))

		   	# read task status
		   	if a.startswith("TASK_STATUS"):
			   	status = getvalue(a)
			
			# read counter value
  			if a.startswith("COUNTERS"):
				#counter = (getvalue(a)).strip("}")
				counter = l[l.find("COUNTERS"):]
				B = counter.split(']')
				for b in B:
					if not b == "":
						if b.startswith("COUNTERS"):
							file_read = getbytevalue(b)
						if b.startswith("[(FILE_BYTES_WRITTEN"):
							file_write = getbytevalue(b)
						if b.startswith("[(HDFS"):
							hdfs_read = getbytevalue(b)
						if b.startswith("[(MAP_OUTPUT_BYTES"):
							map_out = getbytevalue(b)
						if b.startswith("[(MAP_INPUT_BYTES"):
							map_in = getbytevalue(b)
						if b.startswith("[(REDUCE_SHUFFLE_BYTES"):
							reduce_shuffle = getbytevalue(b)

	   	# ignore SETUP and CLEANUP task
	   	if type in ["SETUP", "CLEANUP"]:
		   	continue
		# skip failed attempt
		if status != "SUCCESS":
			continue
		
	   	if type == "MAP":
			if not mhosts.has_key(id):
				mhosts[id] = {}
				mhosts[id]["file_read"] = 0
				mhosts[id]["file_write"] = 0
				mhosts[id]["hdfs_read"] = 0
				mhosts[id]["map_out"] = 0
				mhosts[id]["map_in"] = 0

			mhosts[id]["file_read"] += file_read
			mhosts[id]["file_write"] += file_write
			mhosts[id]["hdfs_read"] += hdfs_read
			mhosts[id]["map_out"] += map_out
			mhosts[id]["map_in"] += map_in

	   	if type == "REDUCE":
			if not rhosts.has_key(id):
				rhosts[id] = {}
				rhosts[id]["file_read"] = 0
				rhosts[id]["file_write"] = 0
				rhosts[id]["reduce_shuffle"] = 0

			rhosts[id]["file_read"] += file_read
			rhosts[id]["file_write"] += file_write
			rhosts[id]["reduce_shuffle"] += reduce_shuffle


mf = open('maptasks.dat', 'w')
rf = open('reducetasks.dat', 'w')

for k,v in mhosts.items():
	mf.write(str(k)+' '+str(v["file_read"])+' '+str(v["file_write"])+' '+str(v["hdfs_read"])+' '+str(v["map_in"])+' '+str(v["map_out"])+'\n')
mf.close()

for k,v in rhosts.items():
	rf.write(str(k)+' '+str(v["file_read"])+' '+str(v["file_write"])+' '+str(v["reduce_shuffle"])+'\n')
rf.close()
